# -*- coding: utf-8 -*-

import commonlib
import traceback
import os
import time, datetime, workerpool
import html2text
from MongoModel import MongoModel
from lxml import etree
from MongoDbLog import MongoDbLog
from pymongo.objectid import ObjectId

PREFIX = '/diadiem/ciao/'
MONGO_SERVER = 'localhost'
MONGO_PORT = 27017

LOG = MongoDbLog('songvang_ciao', 'debug')

class Task(workerpool.Job):
    
    def __init__(self, url, cityId, cat, func):
        self.url = url
        self.cityId = cityId
        self.func = func
        self.cat = cat
        
    def run(self):
        self.func(self.url, self.cat, self.cityId)

class CiaoVn(MongoModel):
    
    categories = {
            '-mua_sam': 'Mua sắm',
            '-nha_hang_-_quan-an': 'Nhà hàng - Quán ăn',
            '-y_te': 'Y tế',
            '-thuc_pham': 'Thực phẩm',
            '-khach_san_-_du_lich': 'Khách sạn - Du lịch',
            '-lam_dep_-_spa': 'Làm đẹp - Spa',
            '-nghe_thuat_-_giai_tri': 'Nghệ thuật - giải trí',
            '-the_thao': 'Thể thao',
            '-giao_duc': 'Giáo dục',
            '-tu_van_-_moi_truong': 'Tư vấn - Môi trường',
            '-kien_truc_-_noi_that': 'Kiến trúc - Nội thất',
            '-oto_-_xe_may_-_xe_dap': 'Ô tô - Xe máy - Xe đạp',
            '-dien_tu_-_dien_may': 'Điện tử - Điện máy'
            }
    
    cities = {
            'ha_noi': 'Hà Nội',
            'tp_hcm': 'TP Hồ Chí Minh',
            'da_nang': 'Đà Nẵng',
            'quang_ninh': 'Quảng Ninh',
            'hue': 'Huế',
            'can_tho': 'Cần Thơ',
            'dong_nai': 'Đồng Nai',
            'hai_phong': 'Hải Phòng',
            'vung_tau': 'Vũng Tàu',
            'khanh_hoa': 'Khánh Hòa',
            'binh_duong': 'Bình Dương'
            }
    
    __url = 'http://ciao.vn/{city}/{cat}'
    
    def __init__(self, host='localhost', port=27017):
        MongoModel.__init__(self, host, port)
        self.db = self.connection['songvang']
    
    def getPicturesShop(self, url):
        url += '/hinh_anh'
        LOG.debug('getPicturesShop(%s)' % url)
        data = []
        try:
            tree = commonlib.getXMLTree(url)
            for elem in tree.xpath("//ul[@class='pkg img-gallery']/li/a"):
                pic = commonlib.urlJoin(url, commonlib.getAttribText(elem, 'href'))
                fn = self.downloadImage(pic)
                if fn != '': data.append(fn)
            return data
        except:
            LOG.error(traceback.format_exc())
        return data
    
    def downloadImage(self, url, **args):
        img = commonlib.downloadImage(url, downloadPath, prefix='')
        print('INFO: image downloaded: %s' % img)
        return img
    
    def detailShop(self, url, args):
        LOG.debug("detailShop(%s, %s)" % (url, args))
        tree = commonlib.getXMLTree(url)
        shopName, address, lat, lng, district, city, cat = '', '', '', '', '', '', ''
        for i, v in args.items():
            exec("%s = '%s'" % (i, v))
        dt = tree.xpath("//label[contains(., 'Điện thoại')]".decode('utf-8'))
        phone = commonlib.getElementText(dt, text=0)
        
        # lay phan gioi thieu shop
        introText = ''
        try:
            for introNode in tree.xpath("//strong[contains(., 'Giới thiệu')]/following-sibling::*[1]".decode('utf-8')):
                h = html2text._html2text()
                h.feed(etree.tostring(introNode, method='html', pretty_print=True, encoding='utf-8').decode('utf-8'))
                h.feed('')
                introText += commonlib.encodeUTF8Str(h.close())
        except:
            LOG.error(traceback.format_exc())
        
        commonlib.replaceStr(r'\t|\n', '', introText).strip()
        
        images = self.getPicturesShop(url)
        
        data = {
            'id': commonlib.getCRC32Unsign("%s%s" % (shopName, address)),
            'shopName': shopName,
            'address': address,
            'lat': lat,
            'lng': lng,
            'phone': phone,
            'images': images,
            'intro': introText,
            'district': district,
            'city': city,
            'lastupdate': datetime.datetime.now(),
            'timestamp': time.time()
            }
        
        collection = self.db[commonlib.replaceStr(r'[^a-z]', '', cat)]
        isExist = collection.find_one({'id': data['id']})
        if isExist:
            data['_id'] = ObjectId(isExist['_id'])
        collection.save(data)
        
        print('shopName: %s' % shopName)
        print('address: %s' % address)
        print('lat: {0}, lng: {1}'.format(lat, lng))
        print('phone: %s' % phone)
        print('detail: %s' % introText)
        
    def listShopByDistrict(self, url, district, cityId, cat):
        LOG.debug('listShopByDistrict(%s, %s)' % (url, district))
        try:
            while url != '':
                tree = commonlib.getXMLTree(url)
                if tree == '': break
                for elem in tree.xpath("//ul[@class='pkg destslist']/li"):
                    lat = commonlib.getAttribText(elem.xpath(".//input[@name='gpsb']"), 'value')
                    lng = commonlib.getAttribText(elem.xpath(".//input[@name='gpsc']"), 'value')
                    link = commonlib.urlJoin(url, commonlib.getAttribText(elem.xpath(".//h2/a[@class='loc-name']"), 'href'))
                    shopName = commonlib.getElementText(elem.xpath(".//h2/a[@class='loc-name']"))
                    address = commonlib.getElementText(elem.xpath(".//label[contains(., 'Địa chỉ')]/following-sibling::*[1]".decode('utf-8')))
                    
                    self.detailShop(link, {'shopName': shopName, 'lat':lat, 'lng':lng, 'address':address, 'district': district, 'city': self.cities[cityId], 'cat': cat})
                nextNode = tree.xpath("//div[@class='pl-wrp pkg']/a[@class='pl-current']/following-sibling::*[1]")
                url = commonlib.urlJoin(url, commonlib.getAttribText(nextNode, 'href')) if len(nextNode) > 0 else ''
        except:
            LOG.error(traceback.format_exc())
            
    def processDistrict(self, url, cat, cityId):
        LOG.debug('start processDistrict(%s, %s)' % (url, cityId))
        try:
            tree = commonlib.getXMLTree(url)
            if tree == '': return
            for elem in tree.xpath("//div[@class='pkg blk-brdis']/ul[@class='pkg brcatlist']/li/a"):
                link = commonlib.urlJoin(url, commonlib.getAttribText(elem, 'href'))
                districtName = commonlib.getAttribText(elem, 'title')
                self.listShopByDistrict(link, districtName, cityId, cat)
        except:
            LOG.error(traceback.format_exc())
        
    def processCategory(self, cityId):
        LOG.debug('start processCategory(%s)' % cityId)
        pool = workerpool.WorkerPool(size=5)
        for cat in self.categories.keys():
            url = self.__url.format(city=cityId, cat=cat)
            pool.put(Task(url, cityId, cat, self.processDistrict))
        pool.shutdown()
        pool.wait()
        
    def processCity(self):
#        for iCity in self.cities.keys():
#            self.processCategory(iCity)
        pool = workerpool.WorkerPool(size=3)
        pool.map(self.processCategory, self.cities)
        pool.shutdown()
        pool.wait()
            
    
if __name__ == '__main__':
    
    downloadPath = '/var/www/songvang/uploads/files/'
    LOG.info('start crawler ciao')
    ciao = CiaoVn()
    ciao.processCity()
    LOG.info('finished crawler ciao')
    os._exit(1)
    
    